{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorBoard Debugger notebook\n",
    "---\n",
    "\n",
    "<font color='red'> <h3>Tested with TensorFlow 1.10</h3></font>\n",
    "\n",
    "This notebook introduces how to connect your TensorFlow application the TensorBoard debugger."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debugger_example():\n",
    "    # Copyright 2016 The TensorFlow Authors. All Rights Reserved.\n",
    "    #\n",
    "    # Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "    # you may not use this file except in compliance with the License.\n",
    "    # You may obtain a copy of the License at\n",
    "    #\n",
    "    #     http://www.apache.org/licenses/LICENSE-2.0\n",
    "    #\n",
    "    # Unless required by applicable law or agreed to in writing, software\n",
    "    # distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "    # See the License for the specific language governing permissions and\n",
    "    # limitations under the License.\n",
    "    # ==============================================================================\n",
    "    \"\"\"Demo of the tfdbg curses CLI: Locating the source of bad numerical values.\n",
    "\n",
    "    The neural network in this demo is larged based on the tutorial at:\n",
    "      tensorflow/examples/tutorials/mnist/mnist_with_summaries.py\n",
    "\n",
    "    But modifications are made so that problematic numerical values (infs and nans)\n",
    "    appear in nodes of the graph during training.\n",
    "    \"\"\"\n",
    "\n",
    "    import argparse\n",
    "    import sys\n",
    "    import os\n",
    "\n",
    "    import tensorflow as tf\n",
    "\n",
    "    from tensorflow.examples.tutorials.mnist import input_data\n",
    "    from tensorflow.python import debug as tf_debug\n",
    "    \n",
    "    from hops import tensorboard\n",
    "\n",
    "    IMAGE_SIZE = 28\n",
    "    HIDDEN_SIZE = 500\n",
    "    NUM_LABELS = 10\n",
    "    RAND_SEED = 42\n",
    "    LEARNING_RATE = 0.025\n",
    "    STEPS = 10\n",
    "    \n",
    "    tensorboard_debug_address = tensorboard.interactive_debugger()\n",
    "    \n",
    "    # How you easily get the TensorBoard logdir for summaries\n",
    "    tensorboard_logdir = tensorboard.logdir()\n",
    "\n",
    "    # Import data\n",
    "    mnist = input_data.read_data_sets(os.getcwd(),\n",
    "                                        one_hot=True,\n",
    "                                        fake_data=False)\n",
    "\n",
    "    def feed_dict(train):\n",
    "        if train:\n",
    "            xs, ys = mnist.train.images, mnist.train.labels\n",
    "        else: \n",
    "            xs, ys = mnist.test.images, mnist.test.labels\n",
    "            \n",
    "        return {x: xs, y_: ys}\n",
    "\n",
    "    sess = tf.InteractiveSession()\n",
    "\n",
    "    # Create the MNIST neural network graph.\n",
    "\n",
    "    # Input placeholders.\n",
    "    with tf.name_scope(\"input\"):\n",
    "      x = tf.placeholder(\n",
    "          tf.float32, [None, IMAGE_SIZE * IMAGE_SIZE], name=\"x-input\")\n",
    "      y_ = tf.placeholder(tf.float32, [None, NUM_LABELS], name=\"y-input\")\n",
    "\n",
    "    def weight_variable(shape):\n",
    "      \"\"\"Create a weight variable with appropriate initialization.\"\"\"\n",
    "      initial = tf.truncated_normal(shape, stddev=0.1, seed=RAND_SEED)\n",
    "      return tf.Variable(initial)\n",
    "\n",
    "    def bias_variable(shape):\n",
    "      \"\"\"Create a bias variable with appropriate initialization.\"\"\"\n",
    "      initial = tf.constant(0.1, shape=shape)\n",
    "      return tf.Variable(initial)\n",
    "\n",
    "    def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):\n",
    "      \"\"\"Reusable code for making a simple neural net layer.\"\"\"\n",
    "      # Adding a name scope ensures logical grouping of the layers in the graph.\n",
    "      with tf.name_scope(layer_name):\n",
    "        # This Variable will hold the state of the weights for the layer\n",
    "        with tf.name_scope(\"weights\"):\n",
    "          weights = weight_variable([input_dim, output_dim])\n",
    "        with tf.name_scope(\"biases\"):\n",
    "          biases = bias_variable([output_dim])\n",
    "        with tf.name_scope(\"Wx_plus_b\"):\n",
    "          preactivate = tf.matmul(input_tensor, weights) + biases\n",
    "\n",
    "        activations = act(preactivate)\n",
    "        return activations\n",
    "\n",
    "    hidden = nn_layer(x, IMAGE_SIZE**2, HIDDEN_SIZE, \"hidden\")\n",
    "    logits = nn_layer(hidden, HIDDEN_SIZE, NUM_LABELS, \"output\", tf.identity)\n",
    "    y = tf.nn.softmax(logits)\n",
    "\n",
    "    with tf.name_scope(\"cross_entropy\"):\n",
    "\n",
    "      diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=logits)\n",
    "      with tf.name_scope(\"total\"):\n",
    "        cross_entropy = tf.reduce_mean(diff)\n",
    "\n",
    "    with tf.name_scope(\"train\"):\n",
    "      train_step = tf.train.AdamOptimizer(LEARNING_RATE).minimize(\n",
    "          cross_entropy)\n",
    "\n",
    "    with tf.name_scope(\"accuracy\"):\n",
    "      with tf.name_scope(\"correct_prediction\"):\n",
    "        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "      with tf.name_scope(\"accuracy\"):\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    sess = tf_debug.TensorBoardDebugWrapperSession(\n",
    "            sess, tensorboard_debug_address)\n",
    "\n",
    "    # Add this point, sess is a debug wrapper around the actual Session if\n",
    "    for i in range(STEPS):\n",
    "      \n",
    "      acc = sess.run(accuracy, feed_dict=feed_dict(False))\n",
    "      print(\"Accuracy at step %d: %s\" % (i, acc))\n",
    "\n",
    "      sess.run(train_step, feed_dict=feed_dict(True))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hops import experiment\n",
    "experiment.launch(debugger_example, name='tensorboard debugger', local_logdir=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting to TensorBoard <a class=\"anchor\" id='tensorboard'></a>\n",
    "To find the TensorBoard for the execution, please go back to HopsWorks and follow the arrows in the images below.\n",
    "\n",
    "![Image7-Monitor.png](../../images/jupyter.png)\n",
    "![Image4-LaunchTensorboard.png](../../images/overview.png)\n",
    "![Image8-Tensorboard.png](../../images/tensorboard_debug.png)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 2
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
